package com.guoxiang.bigdata.spiderMain;


import com.guoxiang.bigdata.dao.TieziDao;
import com.guoxiang.bigdata.domain.TieZi;
import com.guoxiang.bigdata.download.impl.BaidutiebaDownload;
import com.guoxiang.bigdata.paser.impl.BaidutiebaParse;
import com.guoxiang.bigdata.scheduler.impl.BaiduTiebaSchedualForRedis;
;
import java.io.IOException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

/**
 *爬虫的主类
 *
 */

public class Spider {
    /**
     *1.下载页面
     *2.解析页面
     *3.存储数据
     */
    public static void main(String[]args)  {

        //创建下载对象和解析器对象
        final BaidutiebaDownload baidutiebaDownload=new BaidutiebaDownload();
        final BaidutiebaParse baidutiebaParse=new BaidutiebaParse();
        final BaiduTiebaSchedualForRedis bdQueue= BaiduTiebaSchedualForRedis.getBdQueue();
        ExecutorService executorService = Executors.newFixedThreadPool(2);
        final TieziDao tdao=new TieziDao();
        //此任务用来下载页面的地址
        executorService.execute(new Runnable() {
            @Override
            public void run() {
                while (true){
                    //先获取页码然后再慢慢去下载
                    Long currentPageNum = bdQueue.getCurrentPageNum();
                    //下一页
                    bdQueue.nextPage();
                    //获取到html页面字符串
                    String html = baidutiebaDownload.download("https://tieba.baidu.com/f?ie=utf-8&kw=%E7%BE%8E%E5%A5%B3&ie=utf-8&pn="+currentPageNum);
                    //解析每一页的帖子的地址
                   baidutiebaParse.parsePageList(html);

                }
            }
        });
        //此任务用来下载解析具体的页面并且获取其标题和图片
        executorService.execute(new Runnable() {
            @Override
            public void run() {
                while (true) {
                    //从调度器(队列)中获取页面的地址 并且从具体页面总中获取标题和图片
                    String pageUrl = bdQueue.pop();//具体某一页的url
                    if (pageUrl != null) {
                        //下载具体的页面
                        //解析具体的页面的标题和图片
                        String pageHtml = baidutiebaDownload.download("https://tieba.baidu.com" + pageUrl);
                        TieZi tiezi=baidutiebaParse.parseCurrentToTiezi(pageHtml);
                        System.out.println(tiezi);
                        //存储数据
                        try {
                            tdao.add(tiezi);
                        } catch (IOException e) {
                            e.printStackTrace();
                        }
                    }

                }
            }
        });
    }
}
